package model;

import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.*;

/**
 * Retrieves web pages from the World Wide Web. The pages
 * are stored so that other Objects can parse and analyze them.
 * 
 * @author Thomas Nunn
 * @author David Everitt
 * @author Dexter Hu
 * @version TCSS 422 Operating Systems Winter 2014
 */
public class PageRetriever implements Runnable {
	
	/**
	 * The number of web pages to retrieve.
	 */
	private static final int NUMBER_TO_RETRIEVE = 10;

	/**
	 * Map of String URLs to jsoup Documents.
	 */
	private volatile Map<String, Document> myDocs;

	/**
	 * The PageParser.
	 */
	private PageParser parser;

	/**
	 * Represents the state of the web crawler, running or stopped.
	 */
	private volatile boolean isRunning = true;

	/**
	 * Retriever's status.
	 */
	boolean isComplete = false;
	
	/**
	 * Constructs a new PageRetriever object
	 * 
	 * @param theURL The URL of the page to be retrieved
	 * @throws  
	 */
	public PageRetriever() {
		super();
		myDocs = new ConcurrentHashMap<>();
	}

	/**
	 * Retrieves an HTML or text document from the World Wide Web. The URL and Document
	 * are stored in Lists.
	 * 
	 * @param URL The URL of the document to get.
	 * @return True if the URL is valid and hasn't been visited already.
	 */
	public synchronized void retrievePage(String URL) {
		
		boolean isGoodURL = true;
		
//		if (!URL.endsWith("/")) {
//			URL = URL + "/";
//		}
		
		if (URL.endsWith("pdf") | URL.endsWith("png")) {
			isGoodURL = false;
		}

		if (isGoodURL) {
			try {

				final Document myDoc = Jsoup.connect(URL).get();
				myDocs.put(URL, myDoc);

			} catch (IOException e) {
			}

		}
	}

	/**
	 * Verifies that a URL is not malformed.
	 * 
	 * @param theURL The URL to validate.
	 * @return True if the URL is valid.
	 */
	public synchronized boolean validateURL(String theURL) {
		boolean retval = false;

		//Pattern p = Pattern.compile("http://(.*?).(htm|html|txt|php)");

		Pattern p = Pattern.compile("(http|https)://(.*?)./");	//MODIFIED

		Matcher m = p.matcher(theURL);
		
		if (m.matches()) {
			retval = true;
		}
		return retval;
	}

	/**
	 * Resets the fields.
	 */
	public void clear() {
		myDocs.clear();
		parser = new PageParser();
		isRunning = true;
		isComplete = false;
	}

	/**
	 * @return The List of HTML and text Documents
	 */
	public synchronized Map<String, Document> getDocList() {
		return myDocs;
	}

	/**
	 * @return Current number of URL's visited
	 */
	public synchronized int getNumberVisited() {
		return myDocs.size();
	}
	
	/**
	 * @return NUMBER_TO_RETRIEVE field
	 */
	public synchronized int getNumberToRetrieve() {
		return NUMBER_TO_RETRIEVE;
	}
	
	/**
	 * @return True if this PageRetriever has been stopped.
	 */
	public synchronized boolean isFinished() {
		return isComplete;
	}

	/**
	 * Stops the run() method.
	 */
	public void stopCrawler() {
		isRunning = false;
		
	}
	
	/**
	 * @return True if the run() methods is still active.
	 */
	public boolean isRunning() {
		return isRunning;
	}

	
	/**
	 * Sets this Object's PageParser
	 * 
	 * @param theParser
	 */
	public void setParser(PageParser theParser) {
		parser = theParser;
	}

	/**
	 * Repeatedly checks the PageParser's list of URLs and
	 * retrieves new web pages until the isRunning field is false.
	 */
	@Override
	public void run() {
		
		while(isRunning) {

			for (String url : parser.getURLs()) {
				if (!myDocs.containsKey(url)) {
					retrievePage(url);
				}
				
				if(!isRunning) {
					break;
				}
			}
		}
		isComplete = true;
	}
}
